 /*
  *@(#)SiteCrawler.java		8 Sept. 2000
  *
  * Copyright: MetaPlus Inc & Inquiry Management Systems Ltd.  2000
  *
  * This software is the proprietary information of MetaPlus Inc & 
  * Inquiry Management Systems Ltd.  
  * Use is subject to license terms.
  */
package com.ims.iaa;

import java.io.*; 
import java.sql.*;
import java.rmi.RemoteException;
import com.ims.iaa.JavaScript.MyFactory;
import org.mozilla.javascript.ContextFactory; 

/**
  * Provides the container of <code>AdSpider</code> objects ( ad search spiders). 
  * Each <code>SiteCrawler</code> can host up to 100 (change to 500) ad spiders. The number of 
  * spiders can be specified in command line argument.
  *
  * @author Anthony Zhang
  *
  * @version 1.00, 25 Sept. 2000
  *
  */
public class SiteCrawler {
	
	static final int MAX_SPIDER_NO = 500;
	
	//hold the spiders
	static AdSpider[] spiderGroup = new AdSpiderImpl[MAX_SPIDER_NO];
	//used for System.out redirect	
	static PrintStream pri;

	static private Thread monitorThread;
	static private Thread proxyThread;
	//default is 300
	//check the spiders every 300 second 	
	static long MONITOR_FREQUENCY = IaaDebug.SearchProcessTimeout*1000; 
	static boolean MONITOR_KILL = IaaDebug.TimeoutKill;

	// spiders which are function normally
	static int activeSpidersNumber = 0;
	//startup spider number , can be specified in command line
	static int spiderNumber = 50;
	
	static int serverPort = IaaDebug.ServerPort;
    static String serverName = IaaDebug.ServerHost;
    
    private static int m_iBatchIndex = 1; 
	
	public static void main( String[] args ) {
	
	  // we need the properties file loaded first
	  try  {
		  Class.forName("com.ims.iaa.IaaDebug");
		  Class.forName("com.ims.iaa.SpecialExceptionLog");
		  //Class.forName("com.ims.iaa.JavaScript.MyFactory");
	  } catch (Exception e)  {
	  }
	  
	  switch ( args.length ) {

		case 3 :
			try  { // try to get serverPort from command line
				serverPort = Integer.parseInt( args [2] );
			}catch (Exception e ) {
				usageInfo();
				return;
			}		
		case 2:
			try  { // try to get serverName from command line
				serverName = args [1];
			}catch (Exception e ) {
				usageInfo();
				return;
			}
			break;
		case 1:
			try  { // try to get spiderNumber from command line
				spiderNumber = Integer.parseInt( args [0] );
				if (spiderNumber > MAX_SPIDER_NO || spiderNumber <= 0) {
					spiderNumber = 50;
				}
			}catch (Exception e ) {
				usageInfo();
				return;
			}
		case 0: 
			break;
		default:
			usageInfo();
			
	  }
	  	String sLocalName = getMachineName();
//++ Add by ShengMeng Bai for proxy server at 2004-09-24
	  //modify by ShengMeng bai at 2005-04-29
	  	if(IaaDebug.m_bRandomProxy){
		  	startProxyThread(sLocalName);
		  	/*
			System.getProperties().put("proxySet", "true");
			System.getProperties().put("proxyHost", IaaDebug.m_sProxyHost);
			System.getProperties().put("proxyPort", IaaDebug.m_sProxyPort);
			
			String sLog = "Spiders access site through proxy server, proxy setting is:\r\n";
	  		IaaUtil.printlog(sLog);
	  		sLog = "\tHost: " + IaaDebug.m_sProxyHost + "    Port: " + IaaDebug.m_sProxyPort;
			
			if(!IaaDebug.m_sProxyPassword.toLowerCase().equals("anonymous")){
				Authenticator.setDefault( new ProxyAuthenticator(IaaDebug.m_sProxyUserName, IaaDebug.m_sProxyPassword));
				sLog += "    UserName: " + IaaDebug.m_sProxyUserName + "    Password: " + IaaDebug.m_sProxyPassword;
			}
	  		IaaUtil.printlog(sLog);
	  		*/
	  	}
//-- Add by ShengMeng Bai for proxy server at 2004-09-24
		//lookup the Spider Control Center	
	  	SpiderCtl spiderCenter = IaaUtil.getSpiderCenter(serverName, serverPort);
	  	try{
	  		m_iBatchIndex = spiderCenter.registerClient(sLocalName);
	  	}catch(RemoteException e){
	  		System.out.println("Site Crawler: Enter <EXIT> to end.");
	  		return;
	  	}
		for ( int i = 0; i < spiderNumber; i++) {
			spiderGroup[i] = new AdSpiderImpl(serverName, serverPort, m_iBatchIndex);
			activeSpidersNumber++;
		}
		startMonitorThread();

		System.out.println("Site Crawler: Enter <EXIT> to end.");
	
		try {
			pri = creatPrintStream();
			System.setOut(pri);
			System.setErr(pri);
		} catch (Exception e) {
			IaaDebug.systemout.println("Error  redirect output" + e);					
		}		
		
		BufferedReader buff = new BufferedReader( new InputStreamReader(System.in));
		String input = "";
		while (!input.toUpperCase().equals("EXIT")) {
		    try {
		  	 input = buff.readLine();
			 monitorThread.interrupt();
			 //everytime user input check the spiders
			 int number = 0;
		  	 IaaUtil.print("==================Site Crawler: spiders status report============");
		  	 for ( int i = 0; i < spiderNumber; i++) {
			 	AdSpiderImpl ad = (AdSpiderImpl)spiderGroup[i];
				if (ad==null) continue;
				number++;
		  	 	if ( ad.isBusy() )  {
					IaaUtil.print(ad.getSpiderName()+" currently processing search:" 
						+ad.getLocalRequest() );
		  	 	} else  {
			  	 	IaaUtil.print(ad.getSpiderName()+" currently free." );
		  	 	}
				IaaUtil.print(ad.getSpiderName()+" last active time:"
					+ new Timestamp(ad.getLastActiveTime()) );
				IaaUtil.print("");
		  	 }	
			 IaaUtil.print("=================================================================");
			 IaaUtil.print("Site Crawler: "+number
			 	+" spider(s) alive of the total "+spiderNumber+" spider(s)." );
			 IaaUtil.print("=================================================================");
			 if (number==0)  {
			 	IaaUtil.print("!!!!!!!!!!!!!Site Crawler warning: all spiders hangup.!!!!!!!!!!!");
			 	IaaUtil.print("!!!!!!!!!!!!!!!!!Please restart this Site Crawler!!!!!!!!!!!!!!");				
			 }
			 IaaUtil.print("=================================================================");
		  } catch (IOException ioException) { }
		}

		try  {
			for ( int i = 0; i < spiderNumber; i++) {
				spiderGroup[i].notifyTerminate();
			}	
		} catch (Exception e)  {
			IaaUtil.log( "Exit by interrupt error:"+ e );		
		}
		
		IaaUtil.log( "Exit by user" );
		System.exit(0);
	}
	
	private static String getMachineName()  {
		try  {
			String sLocalName = java.net.InetAddress.getLocalHost().getHostName();
			return sLocalName;
		} catch (Exception e)  {
			IaaUtil.log("Error get localhost name" + e);
			return "Unknown";
		}
	}

	private static PrintStream creatPrintStream() {
		ByteArrayOutputStream b = new ByteArrayOutputStream() {
			public void flush() {
				if (size()>2)	{
					String str = toString();
					//22 Nov 2000 temp rem so nothing will redirect to the log file from System.out
					//IaaUtil.log("Redirect from System out: " + str);
				}
				//If buffer size is smaller than 3, the content is discarded
				reset();	
			}
		};
		PrintStream p =  new  PrintStream(b,true);
		return p;
	}	


	private static void usageInfo() {		
		    IaaUtil.log( "usage: SiteCrawler servername spider_no port " );
		    IaaUtil.log( "	 servername: server host name or IP address," 
		    	+ " default is Localhost " );
		    IaaUtil.log( "        spider_no : number of spiders range from 1 to 10,"
		    	+ " default is 1 " );
		    IaaUtil.log( "        port: port number of server rmiregistry port," 
		    	+ " default is 1099 " );		   		    
		    return;
	}
	private static void startProxyThread(String sMachineName){
	    proxyThread = new IaaThread(sMachineName + "_PandomProxy"){
	        public void run(){
	            int iExpireMilliSeconds = IaaDebug.m_iProxyExpireSeconds * 1000;
	            SpiderDBConnection sp = IaaUtil.getSpiderDBConnection(serverName, serverPort);
	            while(true){
	                try {
	                    try {
	                        ProxySetting ps = sp.getProxy(getMachineName(), getName());
	                        if(ps == null){
	                            ProxySetting.unProxy();
	                            IaaUtil.log("Warning, can't get proxy setting form database, will not set the proxy.");
	                        }else{
	                            ps.proxy();
	                        }
                        } catch (RemoteException e) {
                            IaaUtil.printStackTrace(e);
                        }
                        Thread.sleep(iExpireMilliSeconds);
                    } catch (InterruptedException e) {}
	            }
	        }
	    };
	    proxyThread.start();
	}
	
	private static void startMonitorThread()  {
		monitorThread = new IaaThread("SpidersStateMonitor")  {
		
			public void run()  {
				if(IaaDebug.Debug)log("start thread to check spider status");		
				while (true)  {
					//wait for a while when Site Crawler print the report.
					try  {
						Thread.sleep(5000);
					} catch (Exception e)  {
					
					}
					checkSpiders();
					try  {
						Thread.sleep(MONITOR_FREQUENCY);
					} catch (InterruptedException e)  {
						if(IaaDebug.Debug)log("interrupted by user,InterruptedException:"+ e );
					}
				}//end of while
			} // end of run()
			
			private void log(String s)  {
				IaaUtil.print("[Site Crawler Monitor Thread] " + s);
			}
			
		} ; //end of anonymous class
	
		monitorThread.start();
	}
	
	public static void checkSpiders()  {
		for ( int i = 0; i < spiderNumber; i++) {
				AdSpiderImpl ad = (AdSpiderImpl)spiderGroup[i];
				if (ad==null)  {
					continue;
				}
				if ( ad.isBusy() )  {
					IaaUtil.print(ad.getSpiderName()+" currently processing search:" 
						+ad.getLocalRequest() );
					long timeInterval = System.currentTimeMillis()-ad.getLastActiveTime();
					if ((timeInterval > MONITOR_FREQUENCY)&&MONITOR_KILL)  {
						//indicate the spider that it is not good to continue;
						ad.setHealthState(false);
						IaaUtil.print(
							"Spider "+ad.getSpiderName()
							+" stoped response for more than "+timeInterval/1000
							+" seconds.");
						IaaUtil.print(
							"Spider "+ad.getSpiderName()+" exceeds the limit of process timeout which is "
							+IaaDebug.SearchProcessTimeout+" seconds."); 
						IaaUtil.print(
							"Spider "+ad.getSpiderName()+" is now been removed from the Site Crawler.");
							
						spiderGroup[i]=null;
						//Delete by ShengMeng Bai at 2004-09-24, because a new spider will be generate,
						//I think it is sure be wrong.
						//activeSpidersNumber--;
						ad.clearSearch();
						IaaUtil.print("Site Crawler will generate a new spider to replace the old one.");						
						spiderGroup[i] = new AdSpiderImpl(serverName, serverPort, m_iBatchIndex);
					} 
				} 
		}
		if (activeSpidersNumber==0)  {
			IaaUtil.print("=================================================================");	
			IaaUtil.print("!!!!!!!!!!!!!Site Crawler warning: all spiders hangup.!!!!!!!!!!!");
			IaaUtil.print("!!!!!!!!!!!!!!!!!Please restart this Site Crawler!!!!!!!!!!!!!!");				
			IaaUtil.print("=================================================================");	
		}
	}

}